{
 "metadata": {
  "name": ""
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%pylab inline --no-import-all\n",
      "import sys\n",
      "sys.path.extend([u'/home/hadoop/vlmm2/'])\n",
      "import vlmm2.vlmm"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Populating the interactive namespace from numpy and matplotlib\n"
       ]
      }
     ],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "!stop-all.sh\n",
      "!start-all.sh"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Warning: $HADOOP_HOME is deprecated.\r\n",
        "\r\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "stopping jobtracker\r\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "node4: stopping tasktracker\r\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "node3: stopping tasktracker\r\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "node2: stopping tasktracker\r\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "stopping namenode\r\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "node4: stopping datanode\r\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "node3: stopping datanode\r\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "node2: stopping datanode\r\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "node1: stopping secondarynamenode\r\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Warning: $HADOOP_HOME is deprecated.\r\n",
        "\r\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "starting namenode, logging to /usr/local/hadoop/libexec/../logs/hadoop-hadoop-namenode-node1.out\r\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "node4: starting datanode, logging to /usr/local/hadoop/libexec/../logs/hadoop-hadoop-datanode-node4.out\r\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "node3: starting datanode, logging to /usr/local/hadoop/libexec/../logs/hadoop-hadoop-datanode-node3.out\r\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "node2: starting datanode, logging to /usr/local/hadoop/libexec/../logs/hadoop-hadoop-datanode-node2.out\r\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "node1: starting secondarynamenode, logging to /usr/local/hadoop/libexec/../logs/hadoop-hadoop-secondarynamenode-node1.out\r\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "starting jobtracker, logging to /usr/local/hadoop/libexec/../logs/hadoop-hadoop-jobtracker-node1.out\r\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "node4: starting tasktracker, logging to /usr/local/hadoop/libexec/../logs/hadoop-hadoop-tasktracker-node4.out\r\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "node3: starting tasktracker, logging to /usr/local/hadoop/libexec/../logs/hadoop-hadoop-tasktracker-node3.out\r\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "node2: starting tasktracker, logging to /usr/local/hadoop/libexec/../logs/hadoop-hadoop-tasktracker-node2.out\r\n"
       ]
      }
     ],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "vlmm2.vlmm.calc_time(True)\n",
      "vlmm2.vlmm.calc_time(False)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "ERROR:mrjob.fs.hadoop:STDERR: Warning: $HADOOP_HOME is deprecated.\n",
        "\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "ERROR:mrjob.fs.hadoop:STDERR: \n",
        "\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "files size: 100000000\n",
        "mrjob mode: True\n",
        "model length: 1\n",
        "kernel_smooth: False\n",
        "time: 663.152085066"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "files size: 100000000\n",
        "mrjob mode: False\n",
        "model length: 1\n",
        "kernel_smooth: False\n",
        "file /home/hadoop/vlmm2/vlmm2/../data_for_time/train1/pfam.txt\n",
        "time: 126.268488169"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n"
       ]
      },
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 8,
       "text": [
        "126.26848816871643"
       ]
      }
     ],
     "prompt_number": 8
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "vlmm2.vlmm.calc_time(True)\n",
      "vlmm2.vlmm.calc_time(False)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "ERROR:mrjob.fs.hadoop:STDERR: Warning: $HADOOP_HOME is deprecated.\n",
        "\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "ERROR:mrjob.fs.hadoop:STDERR: \n",
        "\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "files size: 245.475883484M\n",
        "mrjob mode: True\n",
        "model length: 2\n",
        "kernel_smooth: False\n",
        "time: 2328.853477"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "files size: 245.475883484M\n",
        "mrjob mode: False\n",
        "model length: 2\n",
        "kernel_smooth: False\n",
        "file /home/hadoop/vlmm2/vlmm2/../data_for_time/train1/pfam.txt\n",
        "time: 584.499224901"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n"
       ]
      },
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 3,
       "text": [
        "584.4992249011993"
       ]
      }
     ],
     "prompt_number": 3
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "vlmm2.vlmm.calc_time(True)\n",
      "vlmm2.vlmm.calc_time(False)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "ERROR:mrjob.fs.hadoop:STDERR: Warning: $HADOOP_HOME is deprecated.\n",
        "\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "ERROR:mrjob.fs.hadoop:STDERR: \n",
        "\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "files size: 245.475883484M\n",
        "mrjob mode: True\n",
        "model length: 2\n",
        "kernel_smooth: False\n",
        "time: 218.901707888"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "files size: 245.475883484M\n",
        "mrjob mode: False\n",
        "model length: 2\n",
        "kernel_smooth: False\n",
        "file /home/hadoop/vlmm2/vlmm2/../data_for_time/train1/pfam.txt\n",
        "time: 620.477015018"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n"
       ]
      },
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 4,
       "text": [
        "620.4770150184631"
       ]
      }
     ],
     "prompt_number": 4
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "vlmm2.vlmm.calc_time(True)\n",
      "vlmm2.vlmm.calc_time(False)\n",
      "# \u89c2\u5bdf\uff0c\u6b64\u65f6\u67094\u4e2aMaP"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "ERROR:mrjob.fs.hadoop:STDERR: Warning: $HADOOP_HOME is deprecated.\n",
        "\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "ERROR:mrjob.fs.hadoop:STDERR: \n",
        "\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "files size: 245.475883484M\n",
        "mrjob mode: True\n",
        "model length: 1\n",
        "kernel_smooth: False\n",
        "time: 131.448464155"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "files size: 245.475883484M\n",
        "mrjob mode: False\n",
        "model length: 1\n",
        "kernel_smooth: False\n",
        "file /home/hadoop/vlmm2/vlmm2/../data_for_time/train1/pfam.txt\n",
        "time: 475.453184843"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n"
       ]
      },
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 3,
       "text": [
        "475.45318484306335"
       ]
      }
     ],
     "prompt_number": 3
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "vlmm2.vlmm.calc_time(True)\n",
      "vlmm2.vlmm.calc_time(False)\n",
      "# \u89c2\u5bdf\uff0c\u6b64\u65f6\u67098\u4e2aMap"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "ERROR:mrjob.fs.hadoop:STDERR: Warning: $HADOOP_HOME is deprecated.\n",
        "\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "ERROR:mrjob.fs.hadoop:STDERR: \n",
        "\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "files size: 490.951766968M\n",
        "mrjob mode: True\n",
        "model length: 1\n",
        "kernel_smooth: False\n",
        "time: 210.790958881"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "files size: 490.951766968M\n",
        "mrjob mode: False\n",
        "model length: 1\n",
        "kernel_smooth: False\n",
        "file /home/hadoop/vlmm2/vlmm2/../data_for_time/train1/pfam.txt\n",
        "time: 962.070239067"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n"
       ]
      },
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 4,
       "text": [
        "962.0702390670776"
       ]
      }
     ],
     "prompt_number": 4
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "vlmm2.vlmm.calc_time(True)\n",
      "vlmm2.vlmm.calc_time(False)\n",
      "# 16\u4e2aMap"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "ERROR:mrjob.fs.hadoop:STDERR: Warning: $HADOOP_HOME is deprecated.\n",
        "\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "ERROR:mrjob.fs.hadoop:STDERR: \n",
        "\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "files size: 981.903533936M\n",
        "mrjob mode: True\n",
        "model length: 1\n",
        "kernel_smooth: False\n",
        "time: 461.763234854"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "files size: 981.903533936M\n",
        "mrjob mode: False\n",
        "model length: 1\n",
        "kernel_smooth: False\n",
        "file /home/hadoop/vlmm2/vlmm2/../data_for_time/train1/pfam.txt\n",
        "time: 2026.52050686"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n"
       ]
      },
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 5,
       "text": [
        "2026.5205068588257"
       ]
      }
     ],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "vlmm2.vlmm.calc_time(True)\n",
      "# vlmm2.vlmm.calc_time(False)\n",
      "# \u6682\u65f6\u5148\u4e0d\u7edf\u8ba1\u8fd9\u4fe1\u606f\n",
      "# 31\u4e2aMap\n",
      "# node1:50030\u9762\u9762\u4e0a\u6709\u5f88\u591a\u4fe1\u606f\uff0c\u53ef\u4ee5\u4f5c\u8bba\u6587\u7528\uff0c\u6bd4\u5982Hadoop\u4f7f\u7528\u65f6\u95f4\u7684\u7edf\u8ba1\n",
      "# files size: 1963.80706787M\n",
      "# mrjob mode: False\n",
      "# model length: 1\n",
      "# kernel_smooth: False\n",
      "# file /home/hadoop/vlmm2/vlmm2/../data_for_time/train1/pfam.txt\n",
      "# time: 3911.90641785"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "ERROR:mrjob.fs.hadoop:STDERR: Warning: $HADOOP_HOME is deprecated.\n",
        "\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "ERROR:mrjob.fs.hadoop:STDERR: \n",
        "\n"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "files size: 1963.80706787M\n",
        "mrjob mode: True\n",
        "model length: 1\n",
        "kernel_smooth: False\n",
        "time: 846.384082079"
       ]
      },
      {
       "ename": "KeyboardInterrupt",
       "evalue": "",
       "output_type": "pyerr",
       "traceback": [
        "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
        "\u001b[1;32m<ipython-input-4-39d7c6833302>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[0mvlmm2\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvlmm\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcalc_time\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mvlmm2\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvlmm\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcalc_time\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[1;31m# 31\u4e2aMap\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
        "\u001b[1;32m/home/hadoop/vlmm2/vlmm2/vlmm.pyc\u001b[0m in \u001b[0;36mcalc_time\u001b[1;34m(use_mrjob)\u001b[0m\n\u001b[0;32m    364\u001b[0m     \u001b[1;31m# \u4f7f\u7528wall time\uff0c\u8fd9\u6837\u624d\u4f1a\u8bb0\u5f55\u8c03\u7528\u5176\u5b83\u7ebf\u7a0b\u7684\u65f6\u95f4\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    365\u001b[0m     \u001b[0mstart\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 366\u001b[1;33m     \u001b[0mvlmm\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrain_dir\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0malphabet\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlength\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mlength\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mp_min\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mp_min\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgamma_min\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mgamma_min\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkernel_smooth\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mkernel_smooth\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0muse_mrjob\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0muse_mrjob\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    367\u001b[0m     \u001b[0mcontinue_time\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m-\u001b[0m \u001b[0mstart\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    368\u001b[0m     \u001b[1;32mprint\u001b[0m \u001b[1;34mu\"time: {0}\"\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcontinue_time\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
        "\u001b[1;32m/home/hadoop/vlmm2/vlmm2/vlmm.pyc\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, trainDir, y, symbols, length, p_min, gamma_min, kernel_smooth, use_mrjob)\u001b[0m\n\u001b[0;32m    163\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfile_name\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfile_names\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    164\u001b[0m             self.tries[x] = self.fit_trie(os.path.join(trainDir, file_name), \n\u001b[1;32m--> 165\u001b[1;33m                 symbols, length, p_min, gamma_min, kernel_smooth, use_mrjob)\n\u001b[0m\u001b[0;32m    166\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtries\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    167\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
        "\u001b[1;32m/home/hadoop/vlmm2/vlmm2/vlmm.pyc\u001b[0m in \u001b[0;36mfit_trie\u001b[1;34m(self, filePath, symbols, length, p_min, gamma_min, kernel_smooth, use_mrjob)\u001b[0m\n\u001b[0;32m    182\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    183\u001b[0m             \u001b[1;32mprint\u001b[0m \u001b[1;34mu\"file {}\"\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfilePath\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 184\u001b[1;33m             \u001b[0mwords\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcalc_frequency\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfilePath\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlength\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    185\u001b[0m         \u001b[0mwords_sum\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msum\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mwords\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitervalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    186\u001b[0m         \u001b[0mlogger\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdebug\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34mu\"words length: {0}\"\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mwords\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
        "\u001b[1;32m/home/hadoop/vlmm2/vlmm2/vlmm.pyc\u001b[0m in \u001b[0;36mcalc_frequency\u001b[1;34m(file_path, length, sep)\u001b[0m\n\u001b[0;32m     97\u001b[0m     \u001b[1;34m\"\"\"Calculate words frequency\"\"\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     98\u001b[0m     \u001b[1;32mfrom\u001b[0m \u001b[0mcollections\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mCounter\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 99\u001b[1;33m     \u001b[0mcounter\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mCounter\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0miter_words\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfile_path\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlength\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msep\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    100\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mcounter\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    101\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
        "\u001b[1;32m/usr/lib/python2.7/collections.pyc\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, iterable, **kwds)\u001b[0m\n\u001b[0;32m    417\u001b[0m         '''\n\u001b[0;32m    418\u001b[0m         \u001b[0msuper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mCounter\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__init__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 419\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0miterable\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    420\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    421\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__missing__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
        "\u001b[1;32m/usr/lib/python2.7/collections.pyc\u001b[0m in \u001b[0;36mupdate\u001b[1;34m(self, iterable, **kwds)\u001b[0m\n\u001b[0;32m    498\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    499\u001b[0m                 \u001b[0mself_get\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 500\u001b[1;33m                 \u001b[1;32mfor\u001b[0m \u001b[0melem\u001b[0m \u001b[1;32min\u001b[0m \u001b[0miterable\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    501\u001b[0m                     \u001b[0mself\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0melem\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself_get\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0melem\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m+\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    502\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mkwds\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
        "\u001b[1;32m/home/hadoop/vlmm2/vlmm2/vlmm.pyc\u001b[0m in \u001b[0;36miter_words\u001b[1;34m(file_path, length, sep)\u001b[0m\n\u001b[0;32m     92\u001b[0m                     \u001b[1;32mif\u001b[0m \u001b[0mx\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0mi\u001b[0m \u001b[1;33m>\u001b[0m \u001b[0mlineLength\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     93\u001b[0m                         \u001b[1;32mbreak\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 94\u001b[1;33m                     \u001b[1;32myield\u001b[0m \u001b[0mline\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m:\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     95\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     96\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mcalc_frequency\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfile_path\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlength\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msep\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
        "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "files size: 1963.80706787M\n",
        "mrjob mode: False\n",
        "model length: 1\n",
        "kernel_smooth: False\n",
        "file /home/hadoop/vlmm2/vlmm2/../data_for_time/train1/pfam.txt\n"
       ]
      }
     ],
     "prompt_number": 4
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}